package main

import (
	"errors"
	"fmt"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"time"
)


//发起请求，获取网页内容
func HttpGet(url string) (result string, err error) {
	client:=&http.Client{}
	res,err1:=http.NewRequest("GET",url,nil)
	res.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 Edg/90.0.818.62")
	if err1 != nil {
		err = err1
		return
	}
	resp,err2:=client.Do(res)
	defer resp.Body.Close()
	if err2 != nil {
		err = err2
		return
	}
	//fmt.Println(resp)
	if resp.StatusCode != 200 {
		//fmt.Println("StatusCode:",resp.StatusCode)
		err=errors.New("错误码:"+strconv.Itoa(resp.StatusCode))
		return
	}
	//读取网页内容
	buf := make([]byte, 4*1024)
	for {
		n, _:= resp.Body.Read(buf)
		if n == 0 {
			break
		}
		result += string(buf[:n])  //累加读取的内容
	}
	return
}

func SpiderPage(i int, page chan int) {
	// 明确爬取的url
	url := "https://movie.douban.com/top250?start=" + strconv.Itoa((i-1)*25) + "&filter="

	time.Sleep(1 * time.Second)       // 为防止访问IP 被网站屏闭，降低爬取频率

	// 开始爬取页面内容，将结果保存至 result。
	result, err := HttpGet(url)
	if err != nil {
		fmt.Println("HttpGet err = ", err)
		return
	}
	//fmt.Println(result)
	// 解析、编译正则表达式。—— 评价人数
	ret1 := regexp.MustCompile(`<span>(?s:(\d*?))人评价</span>`)
	// 取关键信息 评价人数
	pepoleCount := ret1.FindAllStringSubmatch(result, -1)

	// 解析、编译正则表达式。—— 评分
	patternScore := `<span class="rating_num" property="v:average">(?s:(.*?))</span>`
	ret2 := regexp.MustCompile(patternScore)
	filmScore := ret2.FindAllStringSubmatch(result, -1)

	// 解析、编译正则表达式。—— 电影名称
	patternName := `<img width="100" alt="(?s:(.*?))"`
	ret3 := regexp.MustCompile(patternName)
	filmName := ret3.FindAllStringSubmatch(result, -1)

	// 把内容写入到文件
	Save2File(i, pepoleCount, filmScore, filmName)

	page <- i     //写完一个文件，写i 到channel
}

//把内容写入到文件
func Save2File(i int, pepoleCount, filmScore, filmName [][]string) {
	// 新建文件，每一页保存成一个文件
	f, err := os.Create("第 " + strconv.Itoa(i) + " 页.txt")
	if err != nil {
		fmt.Println("os.Create err = ", err)
		return
	}
	defer f.Close()

	// 写标题
	f.WriteString("电影名称" + "\t\t\t" +  "评分" + "\t\t" + "评价人数" + "\t" + "\n")

	// 写内容
	n := len(pepoleCount)
	//fmt.Println(pepoleCount)
	//fmt.Println(filmName)
	//fmt.Println(filmScore)
	for i := 0; i < n; i++ {
		f.WriteString(filmName[i][1] + "\t\t\t" + filmScore[i][1] + "\t\t" + pepoleCount[i][1] + "\t" + "\n")
	}
}

func DoWork(start, end int) {
	fmt.Printf("准备爬取第 %d 页到 %d 页的网址\n", start, end)
	page := make(chan int)
	for i := start; i <= end; i++ {
		//定义一个函数，爬主页面
		go SpiderPage(i, page)
	}

	for i := start; i <= end; i++ {
		fmt.Printf("第 %d 个页面爬取完成\n", <-page)
	}
}

func main() {
	var start, end int
	fmt.Printf("请输入起始页( >= 1 ):")
	fmt.Scan(&start)
	fmt.Printf("请输入终止页( >= 起始页 ):")
	fmt.Scan(&end)

	DoWork(start, end)
}
